[Autogluon] 신용카드거래 사기탐지

Author

김보람

Published

February 2, 2024

imports

import pandas as pd
import numpy as np
import sklearn
import pickle 
import time 
import datetime
import warnings
warnings.filterwarnings('ignore')
%run ../functions.py
with open('../fraudTrain.pkl', 'rb') as file:
    fraudTrain = pickle.load(file)

시도1(0.3 / 0.05)

auto_amt_nb(fraudTrain, 0.3, 0.05)
model time acc pre rec f1 auc graph_based method throw_rate train_size train_cols train_frate test_size test_frate hyper_params
0 KNeighborsUnif None 0.910922 0.341001 0.840000 0.485082 0.944298 False Auto_not_best 0.3 14014 [amt] 0.407164 6006 0.04995 None
1 KNeighborsDist None 0.908758 0.331978 0.816667 0.472062 0.924922 False Auto_not_best 0.3 14014 [amt] 0.407164 6006 0.04995 None
2 LightGBMXT None 0.964202 0.617729 0.743333 0.674735 0.926993 False Auto_not_best 0.3 14014 [amt] 0.407164 6006 0.04995 None
3 LightGBM None 0.927905 0.395604 0.840000 0.537887 0.959906 False Auto_not_best 0.3 14014 [amt] 0.407164 6006 0.04995 None
4 RandomForestGini None 0.900100 0.313433 0.840000 0.456522 0.946392 False Auto_not_best 0.3 14014 [amt] 0.407164 6006 0.04995 None
5 RandomForestEntr None 0.900100 0.313433 0.840000 0.456522 0.946392 False Auto_not_best 0.3 14014 [amt] 0.407164 6006 0.04995 None
6 CatBoost None 0.927239 0.393468 0.843333 0.536585 0.960017 False Auto_not_best 0.3 14014 [amt] 0.407164 6006 0.04995 None
7 ExtraTreesGini None 0.903596 0.322745 0.846667 0.467341 0.950812 False Auto_not_best 0.3 14014 [amt] 0.407164 6006 0.04995 None
8 ExtraTreesEntr None 0.903263 0.321019 0.840000 0.464516 0.950831 False Auto_not_best 0.3 14014 [amt] 0.407164 6006 0.04995 None
9 NeuralNetFastAI None 0.962870 0.604336 0.743333 0.666667 0.904083 False Auto_not_best 0.3 14014 [amt] 0.407164 6006 0.04995 None
10 XGBoost None 0.921911 0.376642 0.860000 0.523858 0.958862 False Auto_not_best 0.3 14014 [amt] 0.407164 6006 0.04995 None
11 NeuralNetTorch None 0.938228 0.438045 0.836667 0.575029 0.954446 False Auto_not_best 0.3 14014 [amt] 0.407164 6006 0.04995 None
12 LightGBMLarge None 0.940559 0.446729 0.796667 0.572455 0.957917 False Auto_not_best 0.3 14014 [amt] 0.407164 6006 0.04995 None
13 WeightedEnsemble_L2 None 0.928072 0.396875 0.846667 0.540426 0.961589 False Auto_not_best 0.3 14014 [amt] 0.407164 6006 0.04995 None

시도2(0.3 / 0.005)

auto_amt_nb(fraudTrain, 0.3, 0.005)
model time acc pre rec f1 auc graph_based method throw_rate train_size train_cols train_frate test_size test_frate hyper_params
0 KNeighborsUnif None 0.908591 0.050260 0.966667 0.095552 0.976763 False Auto_not_best 0.3 14014 [amt] 0.426431 6006 0.004995 None
1 KNeighborsDist None 0.902098 0.047078 0.966667 0.089783 0.961337 False Auto_not_best 0.3 14014 [amt] 0.426431 6006 0.004995 None
2 LightGBMXT None 0.938228 0.070529 0.933333 0.131148 0.985247 False Auto_not_best 0.3 14014 [amt] 0.426431 6006 0.004995 None
3 LightGBM None 0.923909 0.059794 0.966667 0.112621 0.986175 False Auto_not_best 0.3 14014 [amt] 0.426431 6006 0.004995 None
4 RandomForestGini None 0.890443 0.042274 0.966667 0.081006 0.977549 False Auto_not_best 0.3 14014 [amt] 0.426431 6006 0.004995 None
5 RandomForestEntr None 0.890443 0.042274 0.966667 0.081006 0.977549 False Auto_not_best 0.3 14014 [amt] 0.426431 6006 0.004995 None
6 CatBoost None 0.927572 0.058824 0.900000 0.110429 0.981576 False Auto_not_best 0.3 14014 [amt] 0.426431 6006 0.004995 None
7 ExtraTreesGini None 0.893606 0.042105 0.933333 0.080576 0.978085 False Auto_not_best 0.3 14014 [amt] 0.426431 6006 0.004995 None
8 ExtraTreesEntr None 0.893440 0.042042 0.933333 0.080460 0.978952 False Auto_not_best 0.3 14014 [amt] 0.426431 6006 0.004995 None
9 NeuralNetFastAI None 0.972527 0.142857 0.900000 0.246575 0.954507 False Auto_not_best 0.3 14014 [amt] 0.426431 6006 0.004995 None
10 XGBoost None 0.923077 0.057377 0.933333 0.108108 0.981239 False Auto_not_best 0.3 14014 [amt] 0.426431 6006 0.004995 None
11 NeuralNetTorch None 0.953713 0.089404 0.900000 0.162651 0.980567 False Auto_not_best 0.3 14014 [amt] 0.426431 6006 0.004995 None
12 LightGBMLarge None 0.950383 0.083851 0.900000 0.153409 0.984594 False Auto_not_best 0.3 14014 [amt] 0.426431 6006 0.004995 None
13 WeightedEnsemble_L2 None 0.929903 0.062640 0.933333 0.117400 0.983543 False Auto_not_best 0.3 14014 [amt] 0.426431 6006 0.004995 None

시도3(0.2 / 0.05)

auto_amt_nb(fraudTrain, 0.2, 0.05)
model time acc pre rec f1 auc graph_based method throw_rate train_size train_cols train_frate test_size test_frate hyper_params
0 KNeighborsUnif None 0.943945 0.464052 0.788889 0.584362 0.937773 False Auto_not_best 0.2 21021 [amt] 0.264307 9009 0.04995 None
1 KNeighborsDist None 0.937729 0.431397 0.775556 0.554408 0.917876 False Auto_not_best 0.2 21021 [amt] 0.264307 9009 0.04995 None
2 LightGBMXT None 0.964369 0.617486 0.753333 0.678679 0.932437 False Auto_not_best 0.2 21021 [amt] 0.264307 9009 0.04995 None
3 LightGBM None 0.964147 0.614414 0.757778 0.678607 0.962050 False Auto_not_best 0.2 21021 [amt] 0.264307 9009 0.04995 None
4 RandomForestGini None 0.930958 0.400463 0.768889 0.526636 0.937688 False Auto_not_best 0.2 21021 [amt] 0.264307 9009 0.04995 None
5 RandomForestEntr None 0.930958 0.400463 0.768889 0.526636 0.937688 False Auto_not_best 0.2 21021 [amt] 0.264307 9009 0.04995 None
6 CatBoost None 0.967366 0.650579 0.748889 0.696281 0.939039 False Auto_not_best 0.2 21021 [amt] 0.264307 9009 0.04995 None
7 ExtraTreesGini None 0.932401 0.408092 0.784444 0.536882 0.944522 False Auto_not_best 0.2 21021 [amt] 0.264307 9009 0.04995 None
8 ExtraTreesEntr None 0.932845 0.409988 0.784444 0.538520 0.945090 False Auto_not_best 0.2 21021 [amt] 0.264307 9009 0.04995 None
9 NeuralNetFastAI None 0.965479 0.629423 0.751111 0.684904 0.904984 False Auto_not_best 0.2 21021 [amt] 0.264307 9009 0.04995 None
10 XGBoost None 0.967366 0.650579 0.748889 0.696281 0.957612 False Auto_not_best 0.2 21021 [amt] 0.264307 9009 0.04995 None
11 NeuralNetTorch None 0.968476 0.664032 0.746667 0.702929 0.957463 False Auto_not_best 0.2 21021 [amt] 0.264307 9009 0.04995 None
12 LightGBMLarge None 0.964147 0.614414 0.757778 0.678607 0.961990 False Auto_not_best 0.2 21021 [amt] 0.264307 9009 0.04995 None
13 WeightedEnsemble_L2 None 0.965479 0.629423 0.751111 0.684904 0.904984 False Auto_not_best 0.2 21021 [amt] 0.264307 9009 0.04995 None

시도4(0.2 / 0.005)

auto_amt_nb(fraudTrain, 0.2, 0.005)
model time acc pre rec f1 auc graph_based method throw_rate train_size train_cols train_frate test_size test_frate hyper_params
0 KNeighborsUnif None 0.945499 0.072797 0.844444 0.134039 0.958718 False Auto_not_best 0.2 21021 [amt] 0.283574 9009 0.004995 None
1 KNeighborsDist None 0.941392 0.066427 0.822222 0.122924 0.930221 False Auto_not_best 0.2 21021 [amt] 0.283574 9009 0.004995 None
2 LightGBMXT None 0.977356 0.152838 0.777778 0.255474 0.932297 False Auto_not_best 0.2 21021 [amt] 0.283574 9009 0.004995 None
3 LightGBM None 0.967921 0.113924 0.800000 0.199446 0.969762 False Auto_not_best 0.2 21021 [amt] 0.283574 9009 0.004995 None
4 RandomForestGini None 0.934399 0.058252 0.800000 0.108597 0.943136 False Auto_not_best 0.2 21021 [amt] 0.283574 9009 0.004995 None
5 RandomForestEntr None 0.934399 0.058252 0.800000 0.108597 0.943136 False Auto_not_best 0.2 21021 [amt] 0.283574 9009 0.004995 None
6 CatBoost None 0.975136 0.140562 0.777778 0.238095 0.947925 False Auto_not_best 0.2 21021 [amt] 0.283574 9009 0.004995 None
7 ExtraTreesGini None 0.936619 0.061667 0.822222 0.114729 0.953497 False Auto_not_best 0.2 21021 [amt] 0.283574 9009 0.004995 None
8 ExtraTreesEntr None 0.937285 0.062290 0.822222 0.115806 0.961571 False Auto_not_best 0.2 21021 [amt] 0.283574 9009 0.004995 None
9 NeuralNetFastAI None 0.974914 0.139442 0.777778 0.236486 0.912375 False Auto_not_best 0.2 21021 [amt] 0.283574 9009 0.004995 None
10 XGBoost None 0.977800 0.155556 0.777778 0.259259 0.970798 False Auto_not_best 0.2 21021 [amt] 0.283574 9009 0.004995 None
11 NeuralNetTorch None 0.964591 0.106322 0.822222 0.188295 0.963179 False Auto_not_best 0.2 21021 [amt] 0.283574 9009 0.004995 None
12 LightGBMLarge None 0.976912 0.150215 0.777778 0.251799 0.970329 False Auto_not_best 0.2 21021 [amt] 0.283574 9009 0.004995 None
13 WeightedEnsemble_L2 None 0.967921 0.116352 0.822222 0.203857 0.961582 False Auto_not_best 0.2 21021 [amt] 0.283574 9009 0.004995 None

시도5(0.2 / 0.0005)

auto_amt_nb(fraudTrain, 0.2, 0.0005)
model time acc pre rec f1 auc graph_based method throw_rate train_size train_cols train_frate test_size test_frate hyper_params
0 KNeighborsUnif None 0.947275 0.006289 0.75 0.012474 0.970933 False Auto_not_best 0.2 21021 [amt] 0.285524 9009 0.000444 None
1 KNeighborsDist None 0.942502 0.005769 0.75 0.011450 0.952193 False Auto_not_best 0.2 21021 [amt] 0.285524 9009 0.000444 None
2 LightGBMXT None 0.978688 0.015464 0.75 0.030303 0.925833 False Auto_not_best 0.2 21021 [amt] 0.285524 9009 0.000444 None
3 LightGBM None 0.978688 0.015464 0.75 0.030303 0.992018 False Auto_not_best 0.2 21021 [amt] 0.285524 9009 0.000444 None
4 RandomForestGini None 0.935398 0.005137 0.75 0.010204 0.944919 False Auto_not_best 0.2 21021 [amt] 0.285524 9009 0.000444 None
5 RandomForestEntr None 0.935398 0.005137 0.75 0.010204 0.944919 False Auto_not_best 0.2 21021 [amt] 0.285524 9009 0.000444 None
6 CatBoost None 0.976801 0.014218 0.75 0.027907 0.966991 False Auto_not_best 0.2 21021 [amt] 0.285524 9009 0.000444 None
7 ExtraTreesGini None 0.938062 0.005357 0.75 0.010638 0.957329 False Auto_not_best 0.2 21021 [amt] 0.285524 9009 0.000444 None
8 ExtraTreesEntr None 0.937618 0.005319 0.75 0.010563 0.954761 False Auto_not_best 0.2 21021 [amt] 0.285524 9009 0.000444 None
9 NeuralNetFastAI None 0.978688 0.015464 0.75 0.030303 0.896141 False Auto_not_best 0.2 21021 [amt] 0.285524 9009 0.000444 None
10 XGBoost None 0.976801 0.014218 0.75 0.027907 0.986355 False Auto_not_best 0.2 21021 [amt] 0.285524 9009 0.000444 None
11 NeuralNetTorch None 0.978688 0.015464 0.75 0.030303 0.983120 False Auto_not_best 0.2 21021 [amt] 0.285524 9009 0.000444 None
12 LightGBMLarge None 0.969253 0.014235 1.00 0.028070 0.993073 False Auto_not_best 0.2 21021 [amt] 0.285524 9009 0.000444 None
13 WeightedEnsemble_L2 None 0.968920 0.010638 0.75 0.020979 0.984120 False Auto_not_best 0.2 21021 [amt] 0.285524 9009 0.000444 None

시도6(0.5 / 0.5)

auto_amt_nb(fraudTrain, 0.5, 0.5)
model time acc pre rec f1 auc graph_based method throw_rate train_size train_cols train_frate test_size test_frate hyper_params
0 KNeighborsUnif None 0.873994 0.876047 0.871183 0.873608 0.938715 False Auto_not_best 0.5 8409 [amt] 0.500059 3603 0.499861 None
1 KNeighborsDist None 0.862337 0.866779 0.856191 0.861453 0.908154 False Auto_not_best 0.5 8409 [amt] 0.500059 3603 0.499861 None
2 LightGBMXT None 0.879822 0.884702 0.873404 0.879016 0.953848 False Auto_not_best 0.5 8409 [amt] 0.500059 3603 0.499861 None
3 LightGBM None 0.884263 0.896789 0.868406 0.882370 0.955304 False Auto_not_best 0.5 8409 [amt] 0.500059 3603 0.499861 None
4 RandomForestGini None 0.853178 0.854120 0.851749 0.852933 0.937497 False Auto_not_best 0.5 8409 [amt] 0.500059 3603 0.499861 None
5 RandomForestEntr None 0.853178 0.854120 0.851749 0.852933 0.937497 False Auto_not_best 0.5 8409 [amt] 0.500059 3603 0.499861 None
6 CatBoost None 0.885651 0.904956 0.861743 0.882821 0.956760 False Auto_not_best 0.5 8409 [amt] 0.500059 3603 0.499861 None
7 ExtraTreesGini None 0.859839 0.859600 0.860078 0.859839 0.942604 False Auto_not_best 0.5 8409 [amt] 0.500059 3603 0.499861 None
8 ExtraTreesEntr None 0.859006 0.859366 0.858412 0.858889 0.943666 False Auto_not_best 0.5 8409 [amt] 0.500059 3603 0.499861 None
9 NeuralNetFastAI None 0.860949 0.950763 0.761244 0.845513 0.886771 False Auto_not_best 0.5 8409 [amt] 0.500059 3603 0.499861 None
10 XGBoost None 0.886206 0.900865 0.867851 0.884050 0.956570 False Auto_not_best 0.5 8409 [amt] 0.500059 3603 0.499861 None
11 NeuralNetTorch None 0.880933 0.870011 0.895614 0.882627 0.951755 False Auto_not_best 0.5 8409 [amt] 0.500059 3603 0.499861 None
12 LightGBMLarge None 0.883153 0.900697 0.861188 0.880500 0.959474 False Auto_not_best 0.5 8409 [amt] 0.500059 3603 0.499861 None
13 WeightedEnsemble_L2 None 0.885651 0.901677 0.865630 0.883286 0.956977 False Auto_not_best 0.5 8409 [amt] 0.500059 3603 0.499861 None

시도7(0.5 / 0.05)

auto_amt_nb(fraudTrain, 0.5, 0.05)
model time acc pre rec f1 auc graph_based method throw_rate train_size train_cols train_frate test_size test_frate hyper_params
0 KNeighborsUnif None 0.800722 0.192220 0.933333 0.318786 0.937035 False Auto_not_best 0.5 8409 [amt] 0.692829 3603 0.049958 None
1 KNeighborsDist None 0.794893 0.186308 0.922222 0.309991 0.907297 False Auto_not_best 0.5 8409 [amt] 0.692829 3603 0.049958 None
2 LightGBMXT None 0.812101 0.202395 0.938889 0.333005 0.943885 False Auto_not_best 0.5 8409 [amt] 0.692829 3603 0.049958 None
3 LightGBM None 0.728004 0.150350 0.955556 0.259819 0.953606 False Auto_not_best 0.5 8409 [amt] 0.692829 3603 0.049958 None
4 RandomForestGini None 0.778518 0.172669 0.905556 0.290036 0.934478 False Auto_not_best 0.5 8409 [amt] 0.692829 3603 0.049958 None
5 RandomForestEntr None 0.778518 0.172669 0.905556 0.290036 0.934478 False Auto_not_best 0.5 8409 [amt] 0.692829 3603 0.049958 None
6 CatBoost None 0.789897 0.183315 0.927778 0.306141 0.955237 False Auto_not_best 0.5 8409 [amt] 0.692829 3603 0.049958 None
7 ExtraTreesGini None 0.786567 0.180932 0.927778 0.302811 0.940543 False Auto_not_best 0.5 8409 [amt] 0.692829 3603 0.049958 None
8 ExtraTreesEntr None 0.785457 0.180151 0.927778 0.301716 0.940514 False Auto_not_best 0.5 8409 [amt] 0.692829 3603 0.049958 None
9 NeuralNetFastAI None 0.956980 0.554113 0.711111 0.622871 0.798608 False Auto_not_best 0.5 8409 [amt] 0.692829 3603 0.049958 None
10 XGBoost None 0.777963 0.176409 0.938889 0.297012 0.954753 False Auto_not_best 0.5 8409 [amt] 0.692829 3603 0.049958 None
11 NeuralNetTorch None 0.802387 0.194954 0.944444 0.323194 0.949962 False Auto_not_best 0.5 8409 [amt] 0.692829 3603 0.049958 None
12 LightGBMLarge None 0.714405 0.145363 0.966667 0.252723 0.952816 False Auto_not_best 0.5 8409 [amt] 0.692829 3603 0.049958 None
13 WeightedEnsemble_L2 None 0.774910 0.174407 0.938889 0.294169 0.955795 False Auto_not_best 0.5 8409 [amt] 0.692829 3603 0.049958 None

시도8(0.5 / 0.005)

auto_amt_nb(fraudTrain, 0.5, 0.005)
model time acc pre rec f1 auc graph_based method throw_rate train_size train_cols train_frate test_size test_frate hyper_params
0 KNeighborsUnif None 0.775742 0.021792 1.000000 0.042654 0.953642 False Auto_not_best 0.5 8409 [amt] 0.712094 3603 0.004996 None
1 KNeighborsDist None 0.778240 0.022032 1.000000 0.043114 0.921556 False Auto_not_best 0.5 8409 [amt] 0.712094 3603 0.004996 None
2 LightGBMXT None 0.813211 0.026049 1.000000 0.050776 0.969123 False Auto_not_best 0.5 8409 [amt] 0.712094 3603 0.004996 None
3 LightGBM None 0.761032 0.020478 1.000000 0.040134 0.974709 False Auto_not_best 0.5 8409 [amt] 0.712094 3603 0.004996 None
4 RandomForestGini None 0.763253 0.019563 0.944444 0.038331 0.948024 False Auto_not_best 0.5 8409 [amt] 0.712094 3603 0.004996 None
5 RandomForestEntr None 0.763253 0.019563 0.944444 0.038331 0.948024 False Auto_not_best 0.5 8409 [amt] 0.712094 3603 0.004996 None
6 CatBoost None 0.756869 0.020134 1.000000 0.039474 0.973749 False Auto_not_best 0.5 8409 [amt] 0.712094 3603 0.004996 None
7 ExtraTreesGini None 0.771302 0.021378 1.000000 0.041860 0.956299 False Auto_not_best 0.5 8409 [amt] 0.712094 3603 0.004996 None
8 ExtraTreesEntr None 0.769359 0.021201 1.000000 0.041522 0.956423 False Auto_not_best 0.5 8409 [amt] 0.712094 3603 0.004996 None
9 NeuralNetFastAI None 0.606162 0.012526 1.000000 0.024742 0.957787 False Auto_not_best 0.5 8409 [amt] 0.712094 3603 0.004996 None
10 XGBoost None 0.759645 0.020362 1.000000 0.039911 0.972137 False Auto_not_best 0.5 8409 [amt] 0.712094 3603 0.004996 None
11 NeuralNetTorch None 0.800999 0.024490 1.000000 0.047809 0.973501 False Auto_not_best 0.5 8409 [amt] 0.712094 3603 0.004996 None
12 LightGBMLarge None 0.751596 0.019715 1.000000 0.038668 0.973756 False Auto_not_best 0.5 8409 [amt] 0.712094 3603 0.004996 None
13 WeightedEnsemble_L2 None 0.759645 0.020362 1.000000 0.039911 0.972137 False Auto_not_best 0.5 8409 [amt] 0.712094 3603 0.004996 None

시도9(0.5 / 0.0005)

auto_amt_nb(fraudTrain, 0.5, 0.0005)
model time acc pre rec f1 auc graph_based method throw_rate train_size train_cols train_frate test_size test_frate hyper_params
0 KNeighborsUnif None 0.772412 0.001218 1.0 0.002433 0.970433 False Auto_not_best 0.5 8409 [amt] 0.714116 3603 0.000278 None
1 KNeighborsDist None 0.770747 0.001209 1.0 0.002415 0.940589 False Auto_not_best 0.5 8409 [amt] 0.714116 3603 0.000278 None
2 LightGBMXT None 0.783514 0.001280 1.0 0.002558 0.979733 False Auto_not_best 0.5 8409 [amt] 0.714116 3603 0.000278 None
3 LightGBM None 0.762698 0.001168 1.0 0.002334 0.994170 False Auto_not_best 0.5 8409 [amt] 0.714116 3603 0.000278 None
4 RandomForestGini None 0.755482 0.001134 1.0 0.002265 0.977929 False Auto_not_best 0.5 8409 [amt] 0.714116 3603 0.000278 None
5 RandomForestEntr None 0.755482 0.001134 1.0 0.002265 0.977929 False Auto_not_best 0.5 8409 [amt] 0.714116 3603 0.000278 None
6 CatBoost None 0.762143 0.001166 1.0 0.002328 0.992782 False Auto_not_best 0.5 8409 [amt] 0.714116 3603 0.000278 None
7 ExtraTreesGini None 0.759922 0.001155 1.0 0.002307 0.985702 False Auto_not_best 0.5 8409 [amt] 0.714116 3603 0.000278 None
8 ExtraTreesEntr None 0.761865 0.001164 1.0 0.002326 0.984314 False Auto_not_best 0.5 8409 [amt] 0.714116 3603 0.000278 None
9 NeuralNetFastAI None 0.581460 0.000663 1.0 0.001325 0.996113 False Auto_not_best 0.5 8409 [amt] 0.714116 3603 0.000278 None
10 XGBoost None 0.753816 0.001126 1.0 0.002250 0.990700 False Auto_not_best 0.5 8409 [amt] 0.714116 3603 0.000278 None
11 NeuralNetTorch None 0.764918 0.001179 1.0 0.002356 0.981954 False Auto_not_best 0.5 8409 [amt] 0.714116 3603 0.000278 None
12 LightGBMLarge None 0.756869 0.001140 1.0 0.002278 0.996530 False Auto_not_best 0.5 8409 [amt] 0.714116 3603 0.000278 None
13 WeightedEnsemble_L2 None 0.751596 0.001116 1.0 0.002230 0.993892 False Auto_not_best 0.5 8409 [amt] 0.714116 3603 0.000278 None